{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取代理地址\n",
    "\n",
    "# import \n",
    "\n",
    "import requests\n",
    "import os\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(r'C:/Users/jiangcheng/Documents/Python Scripts/python_learn/module0322/scrapy/proxy')\n",
    "headers={'User-Agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36'}\n",
    "url = 'http://www.xicidaili.com/nn/1'\n",
    "s = requests.get(url,headers = headers)\n",
    "soup = BeautifulSoup(s.text,'lxml') # lxml解析器 其实还有很多种解析器 以后慢慢研究\n",
    "ips = soup.select('#ip_list tr')\n",
    "fp = open('host.txt','w')\n",
    "for i in ips:\n",
    "    try:\n",
    "        ipp = i.select('td')\n",
    "        if len(ipp) == 0:\n",
    "            continue\n",
    "        ip = ipp[1].text\n",
    "        host = ipp[2].text\n",
    "        fp.write(ip)\n",
    "        fp.write('\\t')\n",
    "        fp.write(host)\n",
    "        fp.write('\\n')\n",
    "    except Exception as e:\n",
    "        print('no ip!')\n",
    "fp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'cat' 不是内部或外部命令，也不是可运行的程序\n",
      "或批处理文件。\n"
     ]
    }
   ],
   "source": [
    "!cat ./scrapy/proxy/host.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n",
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "# 代理检验\n",
    "# 借用了上面的import\n",
    "\n",
    "url = 'https://www.baidu.com'\n",
    "fp = open('host.txt','r')\n",
    "ips = fp.readlines()\n",
    "proxys = list()\n",
    "for p in ips:\n",
    "    ip = p.strip('\\n').split('\\t')\n",
    "    proxy = 'http:\\\\' + ip[0] + ':' + ip[1]\n",
    "    proxies = {'proxy':proxy}\n",
    "    proxys.append(proxies) # list 可以使用 append 元素 proxies 是字典\n",
    "for pro in proxys:\n",
    "    try:\n",
    "        s = requests.get(url,proxies = pro)\n",
    "        print(s)\n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将爬虫程序 写成一个函数 然后 主程序进行调用 就可以了 \n",
    "\n",
    "def get_detail(cols,book,sheet,proxys):\n",
    "    index = 1\n",
    "    headers = {'User-Agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36'}\n",
    "    for url in cols:\n",
    "        print(url)\n",
    "        print(random.choice(proxys))\n",
    "        try:\n",
    "            s = requests.get(url, headers=headers, proxies=random.choice(proxys))\n",
    "            \n",
    "            soup = BeautifulSoup(s.text,'lxml')\n",
    "            details = soup.select('.con-txt > li')\n",
    "            person = details[0].text.split(': ')[1]\n",
    "            phone = details[1].text.split(': ')[1]\n",
    "            name = details[2].text.split(': ')[1]\n",
    "            address = details[5].text.split(': ')[1]\n",
    "            write_excel(book,sheet,index,name,address,person,phone)\n",
    "            index = index + 1\n",
    "            print('查询成功！')\n",
    "        except Exception as e:\n",
    "            print('爬取网页失败！')\n",
    "    book.save('./2.xls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
